package us.codecraft.webmagic.lianjia.processor;

import cn.hutool.http.HttpUtil;
import cn.hutool.json.JSONArray;
import cn.hutool.json.JSONUtil;
import org.apache.http.HttpHost;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.ConnectTimeoutException;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.proxy.Proxy;

import java.util.HashMap;
import java.util.Map;

public class ProxyValid {
    /**
     * 批量代理IP有效检测
     *
     * @param proxyIpMap
     * @param reqUrl
     */
    public static Map<String, Integer> checkProxyIp(Map<String, Integer> proxyIpMap, String reqUrl) {
        Map<String, Integer> result = new HashMap<String, Integer>();
        for (String proxyHost : proxyIpMap.keySet()) {
            Integer proxyPort = proxyIpMap.get(proxyHost);

            int statusCode = 0;
            try {
                // 连接超时时间（默认10秒 10000ms） 单位毫秒（ms）
                int connectionTimeout = 5000;
                // 读取数据超时时间（默认30秒 30000ms） 单位毫秒（ms）
                int soTimeout = 15000;
                CloseableHttpClient httpClient = HttpClients.createDefault(); // 创建httpClient实例
                HttpGet httpGet = new HttpGet(reqUrl); // 创建httpget实例
                HttpHost proxy = new HttpHost(proxyHost, proxyPort);
                RequestConfig requestConfig = RequestConfig.custom()
                        .setConnectTimeout(connectionTimeout)//设置连接超时时间
                        .setSocketTimeout(soTimeout)//设置读取超时时间
                        .setProxy(proxy)
                        .build();
                httpGet.setConfig(requestConfig);
                httpGet.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:50.0) Gecko/20100101 Firefox/50.0");
                CloseableHttpResponse response = httpClient.execute(httpGet);// 执行http get请求
                statusCode = response.getStatusLine().getStatusCode();
                response.close();// response关闭
                httpClient.close();// httpClient关闭

                if (statusCode == 200) {
                    result.put(proxyHost, proxyPort);
                }
            } catch (ConnectTimeoutException connectTimeoutException) {
                System.out.println("连接超时，舍弃该代理");
            } catch (Exception e) {
                e.printStackTrace();
            }
        }
        return result;
    }

    public static Proxy[] getKuaidailiValidProxy() {
        Map<String, Integer> proxyIpMap = new HashMap<String, Integer>();
        while (proxyIpMap.size() < 5) {
            ProxyKuaidailiProcessor proxyKuaidailiProcessor = new ProxyKuaidailiProcessor();
            Spider.create(proxyKuaidailiProcessor)
                    .addUrl("https://www.kuaidaili.com/free/intr/")
                    //开启5个线程抓取
                    .thread(1)
                    //启动爬虫
                    .run();
            proxyIpMap.putAll(checkProxyIp(proxyKuaidailiProcessor.proxyIpMap, "https://sh.lianjia.com/"));
        }

        Proxy[] proxies = new Proxy[proxyIpMap.size()];
        int i = 0;
        for (String proxyHost : proxyIpMap.keySet()) {
            Integer proxyPort = proxyIpMap.get(proxyHost);
            proxies[i] = new Proxy(proxyHost, proxyPort);
            i++;
        }
        return proxies;
    }

    public static Map<String, Integer> getXhxProxy() {
        Map<String, Integer> proxyIpMap = new HashMap<String, Integer>();
        while (proxyIpMap.size() > 10) {
            String jsonStr = HttpUtil.get("http://www.xiaohexia.cn/getip.php?num=100&format=json&protocol=1");
            JSONArray array = JSONUtil.parseArray(jsonStr);
            for (int i = 0; i < array.size(); i++) {
                String jsonObject = array.get(i).toString();
                String[] ip = jsonObject.split(":");
                proxyIpMap.put(ip[0], Integer.parseInt(ip[1]));
            }
        }
        return proxyIpMap;
    }

    public static void main(String[] args) {
        Proxy[] proxys = getKuaidailiValidProxy();
//        for (String proxyHost : proxys.keySet()) {
//            Integer proxyPort = proxys.get(proxyHost);
//            System.out.format("%s:%s-->%s\n", proxyHost, proxyPort, "OK");
//        }
    }
}

//https://cuiqingcai.com/5094.html
//http://www.xiaohexia.cn/getip.php?num=30&format=json&protocol=1